package com.spider.ask120.magic.WebMagic;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;

import com.spider.ask120.utils.HttpClientUtil;


import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康百科频道
 * @author Shan
 *
 */
public class Baike implements PageProcessor{
	public static Logger log = Logger.getLogger(Baike.class);
	static String jkbkUrl = "http://m.120ask.com/baike/";
	static String viewRegex = "http://m.120ask.com/baike/art/[0-9]+";
	private String art = "http://m.120ask.com/baike/art/";
	static String listUrl = "/baike/\\w+";
	public static final String UTF_8 = "UTF-8";
	public static final String GBK = "GBK";
	static int pageIndex = 1;
	//判断是否是第一次进入
	static int flag = 1;

	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	
	public Site getSite() { 
		return site;
	}
	
	// process是定制爬虫逻辑的核心接口
	/* (non-Javadoc)
	 * @see us.codecraft.webmagic.processor.PageProcessor#process(us.codecraft.webmagic.Page)
	 */
	public void process(Page page) {
		//进入列表页
		if(page.getUrl().regex(jkbkUrl+"$").match()) { 
			//获取健康百科分类
			List<String> baike = page.getHtml().links().regex(listUrl).all();
			// 去重
			HashSet<String> set = new HashSet<String>(baike);
			baike = new ArrayList<String>();
			baike.addAll(set);
			baike.remove("/baike/art");
			page.addTargetRequests(baike);
			
		}else if(page.getUrl().regex(listUrl+"$").match()){
			boolean flag = true;
			while (flag) {
				//获取到话题页列表 
				Map<String, String> params = new HashMap<String, String>();
				params.put("p", pageIndex + "");
				String listJson = HttpClientUtil.getInstance().sendHttpPost(page.getUrl().toString(), params);
				String[] strings = getStrings(listJson);
				if (null != strings) {
					for (String string : strings) {
						String news = unicodeToUtf8(string)+"}";
						int i = news.indexOf("\"i\":\"");
						int t = news.indexOf("\",\"t");
						String substring = news.substring(i+5, t);
						String url = page.getUrl().toString();
						System.out.println("classification:"+url.substring(url.indexOf("baike/")+6, url.length()));//分类
						System.out.println("id:"+substring);//获取id
						System.out.println("title:"+news.substring(news.indexOf("t\":\"")+4, news.indexOf("\",\"c")));//获取文章标题
						System.out.println("abstract:"+news.substring(news.indexOf("s\":\"")+4, news.indexOf("\"}")));//文章简介
						System.out.println("image:"+news.substring(news.indexOf("c\":\"")+4,news.indexOf("\",\"s")));//文章图片
						System.out.println(art+substring);
						page.addTargetRequest(art + substring);//访问id对应详情页
					}
					pageIndex++;
				}else{
					flag = false;
				}
			}
		}else if(page.getUrl().regex(viewRegex+"$").match()){
			page.putField("title", page.getHtml().xpath("//div[@class='question']/h1/text()"));
			page.putField("contents", page.getHtml().xpath("//div[@class='questionList clears']/ul/li/a/text()").all());
			page.putField("pageContent", page.getHtml().xpath("//div[@class='answer']/div/html()").all());
		}
		
	}
	/**
	 * 根据字符串获取字符串数组
	 * @param str
	 * @return
	 */
	public String[] getStrings(String str){
		String [] strings = null;
		if(null != str && !str.isEmpty()){
			String string = str.substring(2, str.length()-1);
			strings = string.split("},");
		}
		return strings;
	}
	/**
	 * unicode转utf8
	 * @param theString
	 * @return
	 */
	public static String unicodeToUtf8(String theString) {
		if (null == theString || theString.isEmpty()) {
			return null;
		}
        char aChar;
        int len = theString.length();
        StringBuffer outBuffer = new StringBuffer(len);
        for (int x = 0; x < len;) {
            aChar = theString.charAt(x++);
            if (aChar == '\\') {
                aChar = theString.charAt(x++);
                if (aChar == 'u') {
                    // Read the xxxx
                    int value = 0;
                    for (int i = 0; i < 4; i++) {
                        aChar = theString.charAt(x++);
                        switch (aChar) {
                        case '0':
                        case '1':
                        case '2':
                        case '3':
                        case '4':
                        case '5':
                        case '6':
                        case '7':
                        case '8':
                        case '9':
                            value = (value << 4) + aChar - '0';
                            break;
                        case 'a':
                        case 'b':
                        case 'c':
                        case 'd':
                        case 'e':
                        case 'f':
                            value = (value << 4) + 10 + aChar - 'a';
                            break;
                        case 'A':
                        case 'B':
                        case 'C':
                        case 'D':
                        case 'E':
                        case 'F':
                            value = (value << 4) + 10 + aChar - 'A';
                            break;
                        default:
                            throw new IllegalArgumentException(
                                    "Malformed   \\uxxxx   encoding.");
                        }
                    }
                    outBuffer.append((char) value);
                } else {
                    if (aChar == 't')
                        aChar = '\t';
                    else if (aChar == 'r')
                        aChar = '\r';
                    else if (aChar == 'n')
                        aChar = '\n';
                    else if (aChar == 'f')
                        aChar = '\f';
                    outBuffer.append(aChar);
                }
            } else
                outBuffer.append(aChar);
        }
        return outBuffer.toString();
    }

	public static void main(String[] args) {
		Spider.create(new Baike()).addUrl(jkbkUrl).thread(1).run();
//		String str = "[{'a':'b'},{'c':'d‘’'}]";
//		JSONArray ja = JSONArray.fromObject(str);
//		System.out.println(ja.toString());
	}
}
